{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Keras Callbacks\n",
    "- Keras Callbacks provide useful tools to babysit training process\n",
    "    - ModelCheckpoint\n",
    "    - Earlystopping\n",
    "    - ReduceLROnPlateau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from sklearn.datasets import load_digits\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keras.models import Sequential\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from keras import optimizers\n",
    "from keras.callbacks import *\n",
    "from keras.layers import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = load_digits()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_data = data.images\n",
    "y_data = data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_data, y_data, test_size = 0.3, random_state = 777)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# reshaping X data => flatten into 1-dimensional\n",
    "X_train = X_train.reshape((X_train.shape[0], -1))\n",
    "X_test = X_test.reshape((X_test.shape[0], -1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# converting y data into categorical (one-hot encoding)\n",
    "y_train = to_categorical(y_train)\n",
    "y_test = to_categorical(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1257, 64)\n",
      "(540, 64)\n",
      "(1257, 10)\n",
      "(540, 10)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. ModelCheckpoint\n",
    "- **ModelCheckpoint** is used to 'checkpoint' model results on training\n",
    "- Oftentimes, it is used to save only best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_model():\n",
    "    model = Sequential()\n",
    "    model.add(Dense(100, input_shape = (X_train.shape[1],)))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(100))\n",
    "    model.add(Activation('relu'))\n",
    "    model.add(Dense(y_train.shape[1]))\n",
    "    model.add(Activation('sigmoid'))\n",
    "    \n",
    "    model.compile(optimizer = 'Adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = create_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating callbacks list\n",
    "- ModelCheckpoint instances are stored in list and passed on when training "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "callbacks = [ModelCheckpoint(filepath = 'saved_model.hdf5', monitor='val_acc', verbose=1, mode='max')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1257 samples, validate on 540 samples\n",
      "Epoch 1/10\n",
      " 500/1257 [==========>...................] - ETA: 2s - loss: 3.5376 - acc: 0.0460Epoch 00000: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 1s - loss: 3.1724 - acc: 0.0597 - val_loss: 2.4454 - val_acc: 0.0944\n",
      "Epoch 2/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.4252 - acc: 0.1060Epoch 00001: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 2.3155 - acc: 0.1138 - val_loss: 2.1753 - val_acc: 0.1222\n",
      "Epoch 3/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.1647 - acc: 0.1180Epoch 00002: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 2.1493 - acc: 0.1337 - val_loss: 2.1191 - val_acc: 0.1463\n",
      "Epoch 4/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.1091 - acc: 0.1780Epoch 00003: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 2.0974 - acc: 0.1718 - val_loss: 2.0719 - val_acc: 0.1722\n",
      "Epoch 5/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.0500 - acc: 0.1840Epoch 00004: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 2.0414 - acc: 0.1862 - val_loss: 2.0102 - val_acc: 0.1685\n",
      "Epoch 6/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.9848 - acc: 0.1920Epoch 00005: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.9637 - acc: 0.1830 - val_loss: 1.9388 - val_acc: 0.1519\n",
      "Epoch 7/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.9097 - acc: 0.1920Epoch 00006: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.8794 - acc: 0.1766 - val_loss: 1.8934 - val_acc: 0.1481\n",
      "Epoch 8/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.8201 - acc: 0.1880Epoch 00007: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.8146 - acc: 0.1877 - val_loss: 1.8480 - val_acc: 0.1981\n",
      "Epoch 9/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.7587 - acc: 0.2280Epoch 00008: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.7341 - acc: 0.2267 - val_loss: 1.7379 - val_acc: 0.2389\n",
      "Epoch 10/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.6366 - acc: 0.2540Epoch 00009: saving model to saved_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.6024 - acc: 0.2713 - val_loss: 1.5623 - val_acc: 0.2944\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1b3da9a0e80>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs = 10, batch_size = 500, callbacks = callbacks, validation_data = (X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 32/540 [>.............................] - ETA: 0s"
     ]
    }
   ],
   "source": [
    "results = model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.294444444886\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', results[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading saved weights\n",
    "- Saved weights can be loaded and used without further training\n",
    "- This is especially useful when training time is long and model has to be reused a number of times"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "another_model = create_model()\n",
    "another_model.load_weights('saved_model.hdf5')\n",
    "another_model.compile(optimizer = 'Adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 32/540 [>.............................] - ETA: 1s"
     ]
    }
   ],
   "source": [
    "results = another_model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.294444444886\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', results[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Selecting best model\n",
    "- Best model during whole epoch can be selected using ModelCheckpoint\n",
    "    - Set **'save_best_only'** parameter as True\n",
    "- Usually, validation accuracy (val acc) is monitored and used as criterion for best model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "callbacks = [ModelCheckpoint(filepath = 'best_model.hdf5', monitor='val_acc', verbose=1, save_best_only = True, mode='max')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = create_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1257 samples, validate on 540 samples\n",
      "Epoch 1/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 4.1478 - acc: 0.1220Epoch 00000: val_acc improved from -inf to 0.12778, saving model to best_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 3.5864 - acc: 0.1146 - val_loss: 2.5897 - val_acc: 0.1278\n",
      "Epoch 2/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.5697 - acc: 0.1540Epoch 00001: val_acc improved from 0.12778 to 0.18519, saving model to best_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 2.4084 - acc: 0.1480 - val_loss: 2.1885 - val_acc: 0.1852\n",
      "Epoch 3/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.1478 - acc: 0.1880Epoch 00002: val_acc improved from 0.18519 to 0.27593, saving model to best_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 2.1528 - acc: 0.2235 - val_loss: 2.1291 - val_acc: 0.2759\n",
      "Epoch 4/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.1211 - acc: 0.2720Epoch 00003: val_acc did not improve\n",
      "1257/1257 [==============================] - 0s - loss: 2.1060 - acc: 0.2816 - val_loss: 2.0562 - val_acc: 0.2630\n",
      "Epoch 5/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 2.0596 - acc: 0.3180Epoch 00004: val_acc did not improve\n",
      "1257/1257 [==============================] - 0s - loss: 2.0305 - acc: 0.2856 - val_loss: 1.9511 - val_acc: 0.2463\n",
      "Epoch 6/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.9704 - acc: 0.2740Epoch 00005: val_acc did not improve\n",
      "1257/1257 [==============================] - 0s - loss: 1.9249 - acc: 0.2776 - val_loss: 1.8395 - val_acc: 0.2704\n",
      "Epoch 7/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.8570 - acc: 0.2720Epoch 00006: val_acc improved from 0.27593 to 0.29630, saving model to best_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.8134 - acc: 0.2856 - val_loss: 1.7412 - val_acc: 0.2963\n",
      "Epoch 8/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.7387 - acc: 0.3060Epoch 00007: val_acc improved from 0.29630 to 0.36667, saving model to best_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.6940 - acc: 0.3238 - val_loss: 1.6175 - val_acc: 0.3667\n",
      "Epoch 9/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.5890 - acc: 0.3520Epoch 00008: val_acc improved from 0.36667 to 0.43148, saving model to best_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.5548 - acc: 0.3835 - val_loss: 1.4702 - val_acc: 0.4315\n",
      "Epoch 10/10\n",
      " 500/1257 [==========>...................] - ETA: 0s - loss: 1.4319 - acc: 0.4520Epoch 00009: val_acc improved from 0.43148 to 0.52407, saving model to best_model.hdf5\n",
      "1257/1257 [==============================] - 0s - loss: 1.4049 - acc: 0.4718 - val_loss: 1.2969 - val_acc: 0.5241\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1b3dcf994e0>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs = 10, batch_size = 500, callbacks = callbacks, validation_data = (X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "best_model = create_model()\n",
    "best_model.load_weights('best_model.hdf5')\n",
    "best_model.compile(optimizer = 'Adam', loss = 'categorical_crossentropy', metrics = ['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      " 32/540 [>.............................] - ETA: 1s"
     ]
    }
   ],
   "source": [
    "results = best_model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.524074074074\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', results[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Early stopping\n",
    "- Cease training when model seems to overfit, i.e., target metric has stopped improving for certain epochs\n",
    "- One can set **'patience'** parameter, which denotes number of epochs that model will endure without any improvements\n",
    "    - e.g., if patience = 1, training will stop when metric has stopped improving for 2 epochs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "callbacks = [EarlyStopping(monitor = 'acc', patience = 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = create_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1257 samples, validate on 540 samples\n",
      "Epoch 1/20\n",
      "1257/1257 [==============================] - 0s - loss: 2.9360 - acc: 0.1169 - val_loss: 2.4016 - val_acc: 0.1019\n",
      "Epoch 2/20\n",
      "1257/1257 [==============================] - 0s - loss: 2.3206 - acc: 0.1241 - val_loss: 2.1502 - val_acc: 0.1278\n",
      "Epoch 3/20\n",
      "1257/1257 [==============================] - 0s - loss: 2.1144 - acc: 0.1440 - val_loss: 2.0918 - val_acc: 0.1778\n",
      "Epoch 4/20\n",
      "1257/1257 [==============================] - 0s - loss: 2.0407 - acc: 0.1766 - val_loss: 1.9940 - val_acc: 0.1852\n",
      "Epoch 5/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.9147 - acc: 0.1949 - val_loss: 1.8281 - val_acc: 0.1796\n",
      "Epoch 6/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.7235 - acc: 0.1893 - val_loss: 1.5992 - val_acc: 0.1593\n",
      "Epoch 7/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.5091 - acc: 0.1663 - val_loss: 1.4244 - val_acc: 0.1407\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1b3f6edf390>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# you could see that model stops training after 7 epochs\n",
    "model.fit(X_train, y_train, epochs = 20, batch_size = 500, callbacks = callbacks, validation_data = (X_test, y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Reduce learning rate\n",
    "- In general, it is more desirable to lower down learning rate (learning rate decay) as training proceeds\n",
    "- However, coming up with optimal learning rate decay scheme is not easy\n",
    "- So, one of heuristics would be reducing learning rate when plateau is reached, in other words, when loss stops decreasing for certain number of epochs\n",
    "    - learning rate will be decreased by factor of 'factor' parameter when objective metric has not improved for 'patience' parameter\n",
    "<br>\n",
    "<img src=\"https://i.ytimg.com/vi/s6jC7Wc9iMI/maxresdefault.jpg\" style=\"width: 600px\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# halve learning rate when validation loss has not reduced for more than 5 epochs\n",
    "callbacks = [ReduceLROnPlateau(monitor = 'val_loss', factor = 0.5, patience = 5)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model = create_model()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1257 samples, validate on 540 samples\n",
      "Epoch 1/20\n",
      "1257/1257 [==============================] - 0s - loss: 3.5826 - acc: 0.1002 - val_loss: 2.5216 - val_acc: 0.1111\n",
      "Epoch 2/20\n",
      "1257/1257 [==============================] - 0s - loss: 2.3803 - acc: 0.0986 - val_loss: 2.1329 - val_acc: 0.1981\n",
      "Epoch 3/20\n",
      "1257/1257 [==============================] - 0s - loss: 2.1003 - acc: 0.1742 - val_loss: 2.0654 - val_acc: 0.2685\n",
      "Epoch 4/20\n",
      "1257/1257 [==============================] - 0s - loss: 2.0401 - acc: 0.2586 - val_loss: 2.0159 - val_acc: 0.3093\n",
      "Epoch 5/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.9859 - acc: 0.2991 - val_loss: 1.9479 - val_acc: 0.3278\n",
      "Epoch 6/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.9023 - acc: 0.3039 - val_loss: 1.8557 - val_acc: 0.3093\n",
      "Epoch 7/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.7986 - acc: 0.2928 - val_loss: 1.7457 - val_acc: 0.3000\n",
      "Epoch 8/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.6810 - acc: 0.3039 - val_loss: 1.5981 - val_acc: 0.3185\n",
      "Epoch 9/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.5162 - acc: 0.3254 - val_loss: 1.4144 - val_acc: 0.3944\n",
      "Epoch 10/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.3174 - acc: 0.4304 - val_loss: 1.1952 - val_acc: 0.5389\n",
      "Epoch 11/20\n",
      "1257/1257 [==============================] - 0s - loss: 1.0737 - acc: 0.5776 - val_loss: 0.9279 - val_acc: 0.7074\n",
      "Epoch 12/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.7909 - acc: 0.7414 - val_loss: 0.6827 - val_acc: 0.8000\n",
      "Epoch 13/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.5552 - acc: 0.8488 - val_loss: 0.5141 - val_acc: 0.8315\n",
      "Epoch 14/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.3975 - acc: 0.8886 - val_loss: 0.3771 - val_acc: 0.8833\n",
      "Epoch 15/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.2875 - acc: 0.9165 - val_loss: 0.3144 - val_acc: 0.8963\n",
      "Epoch 16/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.2257 - acc: 0.9308 - val_loss: 0.2923 - val_acc: 0.9000\n",
      "Epoch 17/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.1859 - acc: 0.9531 - val_loss: 0.2460 - val_acc: 0.9241\n",
      "Epoch 18/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.1558 - acc: 0.9602 - val_loss: 0.2082 - val_acc: 0.9463\n",
      "Epoch 19/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.1252 - acc: 0.9618 - val_loss: 0.1883 - val_acc: 0.9370\n",
      "Epoch 20/20\n",
      "1257/1257 [==============================] - 0s - loss: 0.1066 - acc: 0.9690 - val_loss: 0.1854 - val_acc: 0.9296\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1b3f84bfba8>"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, epochs = 20, batch_size = 500, callbacks = callbacks, validation_data = (X_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "540/540 [==============================] - 0s     \n"
     ]
    }
   ],
   "source": [
    "results = model.evaluate(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy:  0.929629630513\n"
     ]
    }
   ],
   "source": [
    "print('Accuracy: ', results[1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
